{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/lamb_optimizer.py:34: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from albert import modeling\n",
    "from albert import optimization\n",
    "from albert import tokenization\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/tokenization.py:240: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:loading sentence piece model\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tokenization.FullTokenizer(\n",
    "      vocab_file='albert-base-2020-04-10/sp10m.cased.v10.vocab', do_lower_case=False,\n",
    "      spm_model_file='albert-base-2020-04-10/sp10m.cased.v10.model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PAD': 0,\n",
       " 'X': 1,\n",
       " 'nsubj': 2,\n",
       " 'cop': 3,\n",
       " 'det': 4,\n",
       " 'root': 5,\n",
       " 'nsubj:pass': 6,\n",
       " 'acl': 7,\n",
       " 'case': 8,\n",
       " 'obl': 9,\n",
       " 'flat': 10,\n",
       " 'punct': 11,\n",
       " 'appos': 12,\n",
       " 'amod': 13,\n",
       " 'compound': 14,\n",
       " 'advmod': 15,\n",
       " 'cc': 16,\n",
       " 'obj': 17,\n",
       " 'conj': 18,\n",
       " 'mark': 19,\n",
       " 'advcl': 20,\n",
       " 'nmod': 21,\n",
       " 'nummod': 22,\n",
       " 'dep': 23,\n",
       " 'xcomp': 24,\n",
       " 'ccomp': 25,\n",
       " 'parataxis': 26,\n",
       " 'compound:plur': 27,\n",
       " 'fixed': 28,\n",
       " 'aux': 29,\n",
       " 'csubj': 30,\n",
       " 'iobj': 31,\n",
       " 'csubj:pass': 32}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:116: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<albert.modeling.AlbertConfig at 0x7fd3b80e0cc0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "albert_config = modeling.AlbertConfig.from_json_file('albert-base-2020-04-10/config.json')\n",
    "albert_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'albert-base-2020-04-10/model.ckpt-400000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 3\n",
    "batch_size = 32\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "_NEG_INF = -1e9\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        training = True,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        model = modeling.AlbertModel(\n",
    "            config=albert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.words,\n",
    "            input_mask=self.mask,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        self._arc_h = arc_h\n",
    "        self._arc_c = arc_c\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        self._type_h = type_h\n",
    "        self._type_c = type_c\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        self.out_arc = out_arc\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.decode_arc = decode_arc\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "#         self.decode_arc_t = tf.transpose(decode_arc, (0, 2, 1))\n",
    "#         sequence_loss_depends = tf.contrib.seq2seq.sequence_loss(logits = self.decode_arc_t,\n",
    "#                                                      targets = self.heads,\n",
    "#                                                      weights = mask)\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:194: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:588: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:1025: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:253: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-4486bc1f4ea9>:61: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From <ipython-input-10-4486bc1f4ea9>:101: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:36: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:41: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "INFO:tensorflow:++++++ warmup starts at step 0, for 29336 steps ++++++\n",
      "INFO:tensorflow:using adamw\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:101: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from albert-base-2020-04-10/model.ckpt-400000\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.015625, 0.15625, 35.86845]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.015625, 0.15625, 156.66893]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 97788/97788 [8:20:21<00:00,  3.26it/s, accuracy=0.899, accuracy_depends=0.858, cost=0.255]   \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:50<00:00,  6.20it/s, accuracy=0.907, accuracy_depends=0.86, cost=0.237] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.371303, training acc: 0.896068, training depends: 0.824540, valid loss: 0.368443, valid acc: 0.895804, valid depends: 0.822702\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 2\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'albert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  69%|██████▉   | 67520/97788 [5:41:46<2:14:14,  3.76it/s, accuracy=0.933, accuracy_depends=0.802, cost=4.35] "
     ]
    }
   ],
   "source": [
    "epoch = 1\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'albert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    t = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "    return t, bert_tokens, [1] * len(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"193pt\" height=\"218pt\"\n",
       " viewBox=\"0.00 0.00 192.50 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 214)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-214 188.5,-214 188.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94,-173.9735C94,-162.1918 94,-146.5607 94,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"97.5001,-133.0033 94,-123.0034 90.5001,-133.0034 97.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"109\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M83.64,-86.9735C76.6677,-74.8418 67.3498,-58.6287 59.4999,-44.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"62.3642,-42.9295 54.3468,-36.0034 56.2951,-46.4175 62.3642,-42.9295\"/>\n",
       "<text text-anchor=\"middle\" x=\"92\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M105.3041,-86.7532C108.7732,-81.0866 112.5772,-74.8047 116,-69 120.611,-61.1804 125.5263,-52.6241 129.9752,-44.7918\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0613,-46.4448 134.9374,-36.0174 126.9682,-42.9989 133.0613,-46.4448\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7fd0245c1438>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"1938pt\" height=\"1001pt\"\n",
       " viewBox=\"0.00 0.00 1938.00 1001.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 997)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-997 1934,-997 1934,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"321\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"419\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M403.46,-956.9735C392.6999,-944.4919 378.2154,-927.6899 366.2365,-913.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"368.7005,-911.2922 359.5201,-906.0034 363.3986,-915.8628 368.7005,-911.2922\"/>\n",
       "<text text-anchor=\"middle\" x=\"398\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"472\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;13 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M415.4807,-956.9915C414.3909,-946.7387 414.6409,-933.9887 420,-924 422.1908,-919.9166 425.0725,-916.2012 428.3577,-912.8434\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"430.8781,-915.2881 436.0828,-906.0597 426.2592,-910.0282 430.8781,-915.2881\"/>\n",
       "<text text-anchor=\"middle\" x=\"435\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;1 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>13&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M465.7278,-906.1293C461.8746,-916.1843 456.4445,-928.6999 450,-939 447.8946,-942.3649 445.4853,-945.737 442.9607,-948.9945\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"440.2017,-946.8393 436.5552,-956.7868 445.6092,-951.2844 440.2017,-946.8393\"/>\n",
       "<text text-anchor=\"middle\" x=\"473\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;4 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>13&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M410.2751,-871.734C388.6037,-865.8047 364.1449,-858.8654 342,-852 301.6838,-839.5011 290.0293,-834.6123 246.6263,-819.2169\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"247.7787,-815.9121 237.1839,-815.8795 245.4459,-822.512 247.7787,-815.9121\"/>\n",
       "<text text-anchor=\"middle\" x=\"362\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"317\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;14 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>13&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M439.8839,-869.9735C416.0877,-856.617 383.4777,-838.3133 357.8304,-823.9177\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"359.5084,-820.8459 349.075,-819.0034 356.0821,-826.9501 359.5084,-820.8459\"/>\n",
       "<text text-anchor=\"middle\" x=\"415\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"472\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;33 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>13&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M472,-869.9735C472,-858.1918 472,-842.5607 472,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"475.5001,-829.0033 472,-819.0034 468.5001,-829.0034 475.5001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"491\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"593\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;37 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>13&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M497.0713,-869.9735C515.242,-856.9086 539.9962,-839.1102 559.8068,-824.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"561.8849,-827.6829 567.9608,-819.0034 557.7984,-821.9994 561.8849,-827.6829\"/>\n",
       "<text text-anchor=\"middle\" x=\"560.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"752\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;39 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>13&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M530.0162,-869.9735C576.7721,-855.4458 642.3646,-835.0653 690.12,-820.227\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"691.213,-823.5525 699.7241,-817.2429 689.1359,-816.8678 691.213,-823.5525\"/>\n",
       "<text text-anchor=\"middle\" x=\"642.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M154.9147,-788.9412C136.6689,-782.8639 115.2421,-774.7116 97,-765 82.9809,-757.5366 68.6137,-747.5499 56.5723,-738.3927\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"58.6835,-735.6008 48.6403,-732.2267 54.3874,-741.1274 58.6835,-735.6008\"/>\n",
       "<text text-anchor=\"middle\" x=\"117.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"99\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;7 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M175.9015,-782.9735C161.7251,-770.2586 142.55,-753.0603 126.8939,-739.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"128.8541,-736.0748 119.0727,-732.0034 124.1802,-741.2859 128.8541,-736.0748\"/>\n",
       "<text text-anchor=\"middle\" x=\"174.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M198.2792,-782.9735C199.7689,-771.1918 201.7452,-755.5607 203.4398,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"206.9416,-742.3634 204.7237,-732.0034 199.9969,-741.4853 206.9416,-742.3634\"/>\n",
       "<text text-anchor=\"middle\" x=\"215\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;8 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M223.5577,-782.9735C243.7088,-769.792 271.226,-751.792 293.0969,-737.4855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"295.025,-740.4066 301.4776,-732.0034 291.193,-734.5486 295.025,-740.4066\"/>\n",
       "<text text-anchor=\"middle\" x=\"292.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M207,-695.9735C207,-684.1918 207,-668.5607 207,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"210.5001,-655.0033 207,-645.0034 203.5001,-655.0034 210.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>8&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M329,-695.9735C329,-684.1918 329,-668.5607 329,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.5001,-655.0033 329,-645.0034 325.5001,-655.0034 332.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"341\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M329,-608.9735C329,-597.1918 329,-581.5607 329,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.5001,-568.0033 329,-558.0034 325.5001,-568.0034 332.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"341\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 10&#45;&gt;11 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>10&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M329,-521.9735C329,-510.1918 329,-494.5607 329,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.5001,-481.0033 329,-471.0034 325.5001,-481.0034 332.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"341\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"329\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M329,-434.9735C329,-423.1918 329,-407.5607 329,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.5001,-394.0033 329,-384.0034 325.5001,-394.0034 332.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"341\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M470.5496,-782.9735C469.6016,-771.1918 468.344,-755.5607 467.2656,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"470.7394,-741.6904 466.4485,-732.0034 463.7619,-742.2519 470.7394,-741.6904\"/>\n",
       "<text text-anchor=\"middle\" x=\"480\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;38 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>39&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M707.5833,-782.9607C694.9632,-777.5064 681.309,-771.2876 669,-765 652.742,-756.6952 635.3275,-746.6064 620.4507,-737.601\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"621.9064,-734.3888 611.5487,-732.1595 618.2555,-740.3613 621.9064,-734.3888\"/>\n",
       "<text text-anchor=\"middle\" x=\"689\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"693\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;41 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>39&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.7765,-782.812C731.1157,-777.255 726.1699,-771.0188 722,-765 716.8303,-757.538 711.7452,-749.0903 707.3125,-741.2476\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"710.3357,-739.4817 702.4425,-732.4143 704.2056,-742.8614 710.3357,-739.4817\"/>\n",
       "<text text-anchor=\"middle\" x=\"742\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"811\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;46 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>39&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M764.2248,-782.9735C772.5312,-770.7252 783.6589,-754.3165 792.9768,-740.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"796.0748,-742.2442 798.7908,-732.0034 790.2814,-738.3153 796.0748,-742.2442\"/>\n",
       "<text text-anchor=\"middle\" x=\"805\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"912\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;61 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>39&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M792.2244,-782.8938C803.8883,-777.3896 816.5597,-771.1612 828,-765 844.0466,-756.358 861.3283,-746.0476 876.02,-736.9661\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"878.1623,-739.7551 884.7987,-731.4963 874.4605,-733.814 878.1623,-739.7551\"/>\n",
       "<text text-anchor=\"middle\" x=\"874.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"1149\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;64 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>39&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M804.3091,-789.5368C877.1531,-773.5735 1008.9367,-744.694 1086.7469,-727.6424\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1087.8719,-730.979 1096.8908,-725.4194 1086.3734,-724.1412 1087.8719,-730.979\"/>\n",
       "<text text-anchor=\"middle\" x=\"988.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"1549\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"1502\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;16 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>15&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1517.8627,-956.8612C1511.8966,-951.8691 1506.4435,-945.8951 1503,-939 1499.5725,-932.1369 1498.35,-924.0789 1498.2329,-916.4206\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1501.732,-916.5129 1498.6548,-906.3748 1494.7381,-916.219 1501.732,-916.5129\"/>\n",
       "<text text-anchor=\"middle\" x=\"1515\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"1714\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;20 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>15&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1583.1881,-956.9735C1608.6301,-943.5587 1643.5368,-925.1533 1670.8926,-910.7294\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1672.6424,-913.7635 1679.8557,-906.0034 1669.3775,-907.5715 1672.6424,-913.7635\"/>\n",
       "<text text-anchor=\"middle\" x=\"1657\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;15 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>16&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1515.7074,-906.2288C1519.5967,-911.7869 1523.674,-918.0136 1527,-924 1531.0831,-931.3493 1534.9369,-939.6081 1538.2438,-947.3046\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1535.0523,-948.7457 1542.1164,-956.6419 1541.5183,-946.0639 1535.0523,-948.7457\"/>\n",
       "<text text-anchor=\"middle\" x=\"1547\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"1375\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;17 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>16&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1475.6855,-869.9735C1456.5286,-856.8503 1430.4,-838.9512 1409.561,-824.6756\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1411.5087,-821.7674 1401.2808,-819.0034 1407.5526,-827.5424 1411.5087,-821.7674\"/>\n",
       "<text text-anchor=\"middle\" x=\"1459\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"1502\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;24 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>16&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1502,-869.9735C1502,-858.1918 1502,-842.5607 1502,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1505.5001,-829.0033 1502,-819.0034 1498.5001,-829.0034 1505.5001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1523.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"1604\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1691.2079,-869.9735C1674.8365,-857.0253 1652.5867,-839.4276 1634.6591,-825.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1636.7774,-822.4616 1626.7629,-819.0034 1632.435,-827.952 1636.7774,-822.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"1674\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"1714\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;21 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>20&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1714,-869.9735C1714,-858.1918 1714,-842.5607 1714,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1717.5001,-829.0033 1714,-819.0034 1710.5001,-829.0034 1717.5001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1726\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"1820\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;25 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>20&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1735.9633,-869.9735C1751.5972,-857.1419 1772.7942,-839.7443 1789.9879,-825.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1792.5555,-828.0531 1798.0649,-819.0034 1788.1145,-822.6422 1792.5555,-828.0531\"/>\n",
       "<text text-anchor=\"middle\" x=\"1786\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"1899\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;23 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>20&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1759.5793,-870.3217C1773.3303,-864.7296 1788.3853,-858.3462 1802,-852 1822.412,-842.4854 1844.6855,-830.8662 1862.7353,-821.1227\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1864.6421,-824.0699 1871.757,-816.2195 1861.2994,-817.9196 1864.6421,-824.0699\"/>\n",
       "<text text-anchor=\"middle\" x=\"1852.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"1365\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 17&#45;&gt;18 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>17&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1372.928,-782.9735C1371.5738,-771.1918 1369.7771,-755.5607 1368.2366,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1371.6885,-741.5383 1367.0694,-732.0034 1364.7342,-742.3376 1371.6885,-741.5383\"/>\n",
       "<text text-anchor=\"middle\" x=\"1383\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"1512\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;30 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>24&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1504.072,-782.9735C1505.4262,-771.1918 1507.2229,-755.5607 1508.7634,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1512.2658,-742.3376 1509.9306,-732.0034 1505.3115,-741.5383 1512.2658,-742.3376\"/>\n",
       "<text text-anchor=\"middle\" x=\"1518.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"1712\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 21&#45;&gt;22 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>21&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1713.5856,-782.9735C1713.3148,-771.1918 1712.9554,-755.5607 1712.6473,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1716.1429,-741.9202 1712.4139,-732.0034 1709.1447,-742.0812 1716.1429,-741.9202\"/>\n",
       "<text text-anchor=\"middle\" x=\"1726\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"1824\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 25&#45;&gt;26 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>25&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1820.8288,-782.9735C1821.3705,-771.1918 1822.0892,-755.5607 1822.7054,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1826.2092,-742.1536 1823.1723,-732.0034 1819.2166,-741.832 1826.2092,-742.1536\"/>\n",
       "<text text-anchor=\"middle\" x=\"1835\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"1454\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1495.7196,-695.8511C1491.0572,-690.2952 1486.1248,-684.0497 1482,-678 1476.9287,-670.562 1471.9947,-662.1215 1467.7172,-654.2782\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1470.807,-652.634 1463.0283,-645.4408 1464.6235,-655.9149 1470.807,-652.634\"/>\n",
       "<text text-anchor=\"middle\" x=\"1502\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"1570\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1524.0176,-695.9735C1532.1832,-683.7252 1543.1223,-667.3165 1552.2823,-653.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1555.3629,-655.2653 1557.9978,-645.0034 1549.5385,-651.3824 1555.3629,-655.2653\"/>\n",
       "<text text-anchor=\"middle\" x=\"1556\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"1824\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1824,-695.9735C1824,-684.1918 1824,-668.5607 1824,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1827.5001,-655.0033 1824,-645.0034 1820.5001,-655.0034 1827.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1836\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"1824\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1824,-608.9735C1824,-597.1918 1824,-581.5607 1824,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1827.5001,-568.0033 1824,-558.0034 1820.5001,-568.0034 1827.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1836\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"1570\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;32 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>31&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1570,-608.9735C1570,-597.1918 1570,-581.5607 1570,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1573.5001,-568.0033 1570,-558.0034 1566.5001,-568.0034 1573.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1607\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M465,-695.9735C465,-684.1918 465,-668.5607 465,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"468.5001,-655.0033 465,-645.0034 461.5001,-655.0034 468.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"502\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 35&#45;&gt;36 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>35&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M465,-608.9735C465,-597.1918 465,-581.5607 465,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"468.5001,-568.0033 465,-558.0034 461.5001,-568.0034 468.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"485\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"600\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;40 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>41&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M669.1225,-695.9222C662.1442,-690.3684 654.6177,-684.1062 648,-678 639.3794,-670.0458 630.4398,-660.8443 622.6457,-652.465\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"625.1914,-650.0626 615.8519,-645.0603 620.0335,-654.7951 625.1914,-650.0626\"/>\n",
       "<text text-anchor=\"middle\" x=\"668.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"693\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;43 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>41&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M693,-695.9735C693,-684.1918 693,-668.5607 693,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"696.5001,-655.0033 693,-645.0034 689.5001,-655.0034 696.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"708\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"811\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M811,-695.9735C811,-684.1918 811,-668.5607 811,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"814.5001,-655.0033 811,-645.0034 807.5001,-655.0034 814.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"839.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"943\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M842.9906,-695.8475C852.4395,-690.2916 862.7307,-684.0469 872,-678 884.888,-669.5924 898.7043,-659.8447 910.6921,-651.1372\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"913.0555,-653.7448 919.0584,-645.0146 908.9215,-648.0958 913.0555,-653.7448\"/>\n",
       "<text text-anchor=\"middle\" x=\"905\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1064\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>64&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1124.9239,-695.8061C1118.2253,-690.3493 1111.1151,-684.1679 1105,-678 1097.4414,-670.3761 1089.8763,-661.4313 1083.3395,-653.1796\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1086.0749,-650.9955 1077.1838,-645.2336 1080.5411,-655.2824 1086.0749,-650.9955\"/>\n",
       "<text text-anchor=\"middle\" x=\"1125\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"1149\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;65 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>64&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1149,-695.9735C1149,-684.1918 1149,-668.5607 1149,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1152.5001,-655.0033 1149,-645.0034 1145.5001,-655.0034 1152.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1169.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"1221\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;83 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>64&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1175.3871,-695.8513C1181.9537,-690.575 1188.6409,-684.4737 1194,-678 1199.8774,-670.9001 1205.0175,-662.3166 1209.1912,-654.2328\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1212.4105,-655.6155 1213.633,-645.0914 1206.1145,-652.5562 1212.4105,-655.6155\"/>\n",
       "<text text-anchor=\"middle\" x=\"1224.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"1330\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;68 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>64&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1201.0094,-697.6766C1216.7226,-692.1098 1233.8249,-685.4005 1249,-678 1265.0799,-670.1583 1281.9601,-659.8927 1296.124,-650.6287\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1298.1702,-653.4714 1304.5605,-645.0206 1294.295,-647.6418 1298.1702,-653.4714\"/>\n",
       "<text text-anchor=\"middle\" x=\"1294\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"642\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M682.4328,-608.9735C675.3211,-596.8418 665.8168,-580.6287 657.8099,-566.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"660.6304,-564.8603 652.5537,-558.0034 654.5915,-568.4004 660.6304,-564.8603\"/>\n",
       "<text text-anchor=\"middle\" x=\"679\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"743\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;44 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>43&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M703.36,-608.9735C710.3323,-596.8418 719.6502,-580.6287 727.5001,-566.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"730.7049,-568.4175 732.6532,-558.0034 724.6358,-564.9295 730.7049,-568.4175\"/>\n",
       "<text text-anchor=\"middle\" x=\"733\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"943\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M943,-608.9735C943,-597.1918 943,-581.5607 943,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"946.5001,-568.0033 943,-558.0034 939.5001,-568.0034 946.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"955\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"761\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;49 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>48&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M898.1337,-521.8225C885.0913,-516.3167 870.8903,-510.1054 858,-504 839.7197,-495.3417 819.8938,-485.0792 802.9586,-476.0382\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"804.3414,-472.808 793.8757,-471.159 801.0288,-478.9746 804.3414,-472.808\"/>\n",
       "<text text-anchor=\"middle\" x=\"870\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;50 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>48&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M917.7325,-521.6446C911.0188,-516.2809 903.9786,-510.1878 898,-504 890.8252,-496.5742 883.8215,-487.7681 877.8213,-479.5811\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"880.6175,-477.4739 871.9666,-471.3573 874.915,-481.5336 880.6175,-477.4739\"/>\n",
       "<text text-anchor=\"middle\" x=\"918.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"943\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;51 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>48&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M943,-521.9735C943,-510.1918 943,-494.5607 943,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"946.5001,-481.0033 943,-471.0034 939.5001,-481.0034 946.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"964.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1051\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;57 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>48&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M967.0634,-521.8816C974.4712,-516.2203 982.6176,-509.909 990,-504 1000.5446,-495.56 1011.9396,-486.1216 1021.9947,-477.6808\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1024.3375,-480.2836 1029.7295,-471.1635 1019.827,-474.9305 1024.3375,-480.2836\"/>\n",
       "<text text-anchor=\"middle\" x=\"1027\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"852\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 51&#45;&gt;52 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>51&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M921.8721,-434.9822C915.4045,-429.325 908.3292,-422.9907 902,-417 893.4506,-408.9079 884.3658,-399.8037 876.3185,-391.5509\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"878.7351,-389.0152 869.2649,-384.2648 873.7057,-393.884 878.7351,-389.0152\"/>\n",
       "<text text-anchor=\"middle\" x=\"914\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"948\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1028.5195,-434.8023C1021.5864,-429.1378 1013.9493,-422.8446 1007,-417 996.9619,-408.5576 986.0721,-399.2015 976.4128,-390.8322\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"978.479,-387.9909 968.634,-384.0758 973.8887,-393.2758 978.479,-387.9909\"/>\n",
       "<text text-anchor=\"middle\" x=\"1027\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1051\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;56 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>57&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1051,-434.9735C1051,-423.1918 1051,-407.5607 1051,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1054.5001,-394.0033 1051,-384.0034 1047.5001,-394.0034 1054.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1079.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"1186\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;59 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>57&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1082.6023,-434.9184C1092.1345,-429.3123 1102.5686,-423.0226 1112,-417 1125.3558,-408.4714 1139.7615,-398.6829 1152.2819,-389.9764\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1154.4235,-392.7497 1160.6126,-384.1504 1150.4118,-387.0132 1154.4235,-392.7497\"/>\n",
       "<text text-anchor=\"middle\" x=\"1158.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"852\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;53 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>52&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M852,-347.9735C852,-336.1918 852,-320.5607 852,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"855.5001,-307.0033 852,-297.0034 848.5001,-307.0034 855.5001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"864\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"852\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 53&#45;&gt;54 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>53&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M852,-260.9735C852,-249.1918 852,-233.5607 852,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"855.5001,-220.0033 852,-210.0034 848.5001,-220.0034 855.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"864\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1130\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;58 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>59&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1174.3968,-347.9735C1166.5128,-335.7252 1155.9509,-319.3165 1147.1067,-305.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1149.9439,-303.5176 1141.5884,-297.0034 1144.0578,-307.3064 1149.9439,-303.5176\"/>\n",
       "<text text-anchor=\"middle\" x=\"1178\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"1241\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1197.396,-347.9735C1205.1393,-335.7252 1215.5126,-319.3165 1224.1987,-305.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1227.2333,-307.3262 1229.6186,-297.0034 1221.3165,-303.5857 1227.2333,-307.3262\"/>\n",
       "<text text-anchor=\"middle\" x=\"1228\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1064\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 62&#45;&gt;63 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>62&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1064,-608.9735C1064,-597.1918 1064,-581.5607 1064,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1067.5001,-568.0033 1064,-558.0034 1060.5001,-568.0034 1067.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1076\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"1220\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;66 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>68&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1307.2079,-608.9735C1290.8365,-596.0253 1268.5867,-578.4276 1250.6591,-564.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1252.7774,-561.4616 1242.7629,-558.0034 1248.435,-566.952 1252.7774,-561.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"1302\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"1331\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;67 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>68&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1330.2072,-608.9735C1330.3426,-597.1918 1330.5223,-581.5607 1330.6763,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1334.1778,-568.043 1330.7931,-558.0034 1327.1783,-567.9624 1334.1778,-568.043\"/>\n",
       "<text text-anchor=\"middle\" x=\"1359.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"1440\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;69 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1365.2643,-608.8867C1374.3202,-603.6101 1383.8106,-597.4998 1392,-591 1401.3334,-583.5922 1410.555,-574.3822 1418.3828,-565.8354\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1421.1067,-568.0394 1425.1429,-558.2434 1415.8788,-563.3843 1421.1067,-568.0394\"/>\n",
       "<text text-anchor=\"middle\" x=\"1431.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"1320\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;70 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>69&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1415.1359,-521.9735C1397.1958,-508.9669 1372.7849,-491.269 1353.1823,-477.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1354.9827,-474.0394 1344.8322,-471.0034 1350.8739,-479.7067 1354.9827,-474.0394\"/>\n",
       "<text text-anchor=\"middle\" x=\"1399\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"1444\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;73 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>69&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1440.8288,-521.9735C1441.3705,-510.1918 1442.0892,-494.5607 1442.7054,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1446.2092,-481.1536 1443.1723,-471.0034 1439.2166,-480.832 1446.2092,-481.1536\"/>\n",
       "<text text-anchor=\"middle\" x=\"1454\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"1576\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;75 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>69&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1468.1793,-521.9735C1488.785,-508.792 1516.9229,-490.792 1539.2871,-476.4855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1541.319,-479.3405 1547.8568,-471.0034 1537.5469,-473.4438 1541.319,-479.3405\"/>\n",
       "<text text-anchor=\"middle\" x=\"1528\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"1322\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;71 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>70&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1320.4144,-434.9735C1320.6852,-423.1918 1321.0446,-407.5607 1321.3527,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1324.8553,-394.0812 1321.5861,-384.0034 1317.8571,-393.9202 1324.8553,-394.0812\"/>\n",
       "<text text-anchor=\"middle\" x=\"1359\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"1442\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;72 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>73&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1443.5856,-434.9735C1443.3148,-423.1918 1442.9554,-407.5607 1442.6473,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1446.1429,-393.9202 1442.4139,-384.0034 1439.1447,-394.0812 1446.1429,-393.9202\"/>\n",
       "<text text-anchor=\"middle\" x=\"1460\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"1576\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;74 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>75&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1576,-434.9735C1576,-423.1918 1576,-407.5607 1576,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1579.5001,-394.0033 1576,-384.0034 1572.5001,-394.0034 1579.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1592\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"1718\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;77 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1605.4225,-434.9735C1627.0325,-421.7336 1656.5767,-403.6326 1679.977,-389.2958\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1681.9168,-392.212 1688.6152,-384.0034 1678.2598,-386.2432 1681.9168,-392.212\"/>\n",
       "<text text-anchor=\"middle\" x=\"1666.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"1674\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>77&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1708.2281,-347.7409C1705.2317,-342.0738 1701.9486,-335.7946 1699,-330 1695.0577,-322.2525 1690.866,-313.793 1687.064,-306.0307\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1690.1911,-304.4579 1682.6629,-297.0029 1683.899,-307.5254 1690.1911,-304.4579\"/>\n",
       "<text text-anchor=\"middle\" x=\"1719\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"1784\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>77&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1731.6752,-347.9735C1741.0556,-335.6085 1753.6525,-319.0036 1764.1357,-305.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1767.0868,-307.0857 1770.3423,-297.0034 1761.5099,-302.8549 1767.0868,-307.0857\"/>\n",
       "<text text-anchor=\"middle\" x=\"1766\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"1738\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;79 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>78&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1774.4688,-260.9735C1768.0543,-248.8418 1759.4818,-232.6287 1752.2599,-218.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1755.2874,-217.2077 1747.519,-210.0034 1749.0992,-220.4797 1755.2874,-217.2077\"/>\n",
       "<text text-anchor=\"middle\" x=\"1784\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"1856\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;80 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>78&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1798.9184,-260.9735C1809.2481,-248.4919 1823.1532,-231.6899 1834.6529,-217.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1837.4213,-219.9388 1841.1007,-210.0034 1832.0286,-215.4758 1837.4213,-219.9388\"/>\n",
       "<text text-anchor=\"middle\" x=\"1862\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"1856\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 80&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>80&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1856,-173.9735C1856,-162.1918 1856,-146.5607 1856,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1859.5001,-133.0033 1856,-123.0034 1852.5001,-133.0034 1859.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1893\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"1856\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 81&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>81&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1856,-86.9735C1856,-75.1918 1856,-59.5607 1856,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1859.5001,-46.0033 1856,-36.0034 1852.5001,-46.0034 1859.5001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1876\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7fd0245bd160>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'albert-base-dependency/model.ckpt'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'albert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:++++++ warmup starts at step 0, for 29336 steps ++++++\n",
      "INFO:tensorflow:using adamw\n",
      "INFO:tensorflow:Restoring parameters from albert-base-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word, training = False)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'albert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 313/313 [00:51<00:00,  6.09it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    1.00000   1.00000   1.00000    627830\n",
      "            X    1.00000   1.00000   1.00000     60741\n",
      "          acl    0.83673   0.81078   0.82355      3192\n",
      "        advcl    0.65863   0.62082   0.63917      1585\n",
      "       advmod    0.94333   0.93545   0.93938      6460\n",
      "         amod    0.89656   0.89594   0.89625      4363\n",
      "        appos    0.80183   0.74289   0.77124      3061\n",
      "         case    0.98074   0.98002   0.98038     10862\n",
      "           cc    0.98250   0.97005   0.97624      3473\n",
      "        ccomp    0.51095   0.40698   0.45307       344\n",
      "     compound    0.88936   0.93235   0.91035     11027\n",
      "compound:plur    0.61538   0.55172   0.58182        29\n",
      "         conj    0.89128   0.87559   0.88336      5112\n",
      "          cop    0.97607   0.96453   0.97026       592\n",
      "        csubj    0.33333   0.12500   0.18182         8\n",
      "          dep    0.63277   0.63456   0.63366       353\n",
      "          det    0.94329   0.91823   0.93059      3840\n",
      "        fixed    0.90071   0.73837   0.81150       172\n",
      "         flat    0.95078   0.96312   0.95691     18492\n",
      "         iobj    0.00000   0.00000   0.00000         1\n",
      "         mark    0.91609   0.92954   0.92276      1703\n",
      "         nmod    0.84561   0.82701   0.83621      4457\n",
      "        nsubj    0.84280   0.84806   0.84542      6891\n",
      "   nsubj:pass    0.81498   0.77772   0.79591      1903\n",
      "       nummod    0.95994   0.95256   0.95624      4427\n",
      "          obj    0.89881   0.91319   0.90594      6128\n",
      "          obl    0.86886   0.84068   0.85454      4965\n",
      "    parataxis    0.51012   0.35097   0.41584       359\n",
      "        punct    0.99626   0.99768   0.99697     20300\n",
      "         root    0.88585   0.91420   0.89980     10000\n",
      "        xcomp    0.71181   0.72865   0.72013      1522\n",
      "\n",
      "     accuracy                        0.98614    824192\n",
      "    macro avg    0.80630   0.77893   0.78998    824192\n",
      " weighted avg    0.98598   0.98614   0.98603    824192\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.821895729831751\n",
      "types accuracy: 0.797527251552637\n",
      "root accuracy: 1.0\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from albert-base-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-80-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 40 variables.\n",
      "INFO:tensorflow:Converted 40 variables to const ops.\n",
      "3731 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('albert-base-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'albert-base-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
